/**
 * @file   Crawl.hpp
 * @author Yuan Liu <liuyuan@net.pku.edu.cn>
 * @date   Tue Aug 28 18:01:54 2007
 * 
 * @brief  
 * 
 * 
 */
#ifndef __CRAWL_HPP
#define __CRAWL_HPP

#include <map>
#include <queue>
#include <vector>
#include <string>
#include <fstream>
#include <pthread.h>

#include "Http.hpp"
#include "Url.hpp"
#include "TianwangFile.hpp"

class Crawl
{
public:
    /**
     * @param threadNum 给出线程池的线程上限
     */
    Crawl(const std::string& crPath, int threadNum = 10);
    ~Crawl();

    //     void ImportUrl(const std::string& crUrl);
    int  ImportTaskQueue(const std::vector<std::queue<Url> >& crTaskQueue);
    int  StartCrawling();

    void fetch();
    std::vector<std::string> GetUrlList();

private:
    static void* StartThread(void* arg);

private:
    std::string   mPath;
    int           mThreadNum;
    int           mCurrentFile;
    int           mCurrentQueue;
    int           mCurrentTWFile;
    TianwangFile  mTWFile;
    std::ofstream mFileStream;

    std::vector<std::queue<Url> > mTaskQueue;
    std::vector<std::string> mUrlList;
};

// struct Para
// {
//     Crawl*   pt;
//     unsigned thread_idx;
// };

#endif
